#!/usr/bin/perl -w
# Proceses wikipedia mcc/mnc page
# Just do a wget of the page http://en.wikipedia.org/wiki/Mobile_Network_Code

use strict;
use Text::Unidecode;
use HTML::Entities;
use Unicode::Normalize;


my $ifile = 'wikipedia.com';
my $ofile = 'wikipedia.com.tsv';

open(FD, "$ifile") or die "$!\n";
open(OFD, '>', $ofile) or die "$!\n";

my $mcc;
my $mnc;
my $brand;
my $operator_name;
my $country;
my @buf;

while (my $line = <FD>) {
    chomp $line;
    #$line = unidecode($line);
    $line = decode_entities($line);

    unless ($line =~ /<tr/) {
        push @buf, $line;
    } else {
        while ($line !~ /<tr.*<\/tr>/) {
            my $tmp = <FD>;
	    #$tmp = unidecode($tmp);
	    $tmp = decode_entities($tmp);
            chomp $tmp;
            $line .= $tmp;
        }
        push @buf, $line;
    }
}
close FD;

foreach my $line (@buf) {

    next if ($line =~ /^$/);
    my @a = split(/<td/, $line);

    # country title header
    if (scalar(@a) == 1) {
	# important here - for now store this as ZZ. Then in make_workflow_tables.pl, 
	# make sure that the associated MCC for this 'ZZ' country is not already taken.

	if ($a[0] =~ /no ISO code/) {
	    $country = 'ZZ';
	    next;
	}
	if ($a[0] =~ /title=\"Edit section:\s*International\s*\">edit<\/a>/) {
	    $country = 'ZZ';
	    next;
	} elsif ($a[0] =~ /title=\"Edit section:\s*.*\s*-\s*(\w+)\s*[^>]*\">edit<\/a>/) {
	    $country = $1;
	    next;
	}
    }

    next if (scalar(@a) < 4);

#     next if ($a[0] !~ /<tr>/);

#     # table header
#     if ($a[0] =~ />MCC</) {
# 	next;
#     }

    

    # Skip question book icon
    next if ($a[1] =~ /title\=File:Question/);
    
    # Contents
    if ($a[1] =~ />(\d+)<\/td>/) {
	$mcc = sprintf("%d", $1);
    } else {
	$mcc = "";
    }
    
    if ($a[2] =~ />(\d+)\W*<\/td>/) {
	$mnc = sprintf("%d", $1);
    } else {
	$mnc = "";
    }

    # brand
    if ($a[3] =~ />(.*)<sup class/) {
	$brand = $1;
    } elsif ($a[3] =~ /<a[^>]+>(.*)<\/a>/) {
	$brand = $1;
    } elsif ($a[3] =~ />(.*)<\/td>/) {
	$brand = $1;
    } else {
	$brand = "";
    }
    # get rid of special html formating.
    $brand =~ s/<sub>|<\/sub>//g;
    if ($brand =~ /[<>]+/) {
	if ($brand =~ /title=\"(.*)\"\s+class/) {
	    $brand = $1;
	    if ($brand =~ /[<>]+/) {
		$brand = "";
	    }
	}
    }

    $a[4] =~ s/\(formerly.*$//;
    if ($a[4] =~ /<a[^>]+>(.*)<\/a><sup class/) {
	$operator_name = $1;
    } elsif ($a[4] =~ /<a[^>]+>(.*)<\/a>/) {
	$operator_name = $1;
    } elsif ($a[4] =~ />(.*)<sup id=/) {
	print "fix\n";
	$operator_name = $1;
    } elsif ($a[4] =~ />(.*)<\/td>/) {
	$operator_name = $1;
    } else {
	$operator_name = "";
    }
    # get rid of special html formating.
    $operator_name =~ s/<sub>|<\/sub>//g;
    
    if ($operator_name =~ /[<>]+/) {
	if ($a[4] =~ /title=\"(.*)\"\s+class/) {
	    $operator_name = $1;
	} elsif ($a[4] =~ />(.*)<sup/) {
	    $operator_name = $1;
	}
    }
    if ($operator_name =~ /[<>]+/) {
	$operator_name = "";
    }


    if ($operator_name eq "" and $brand ne "") {
	$operator_name = $brand;
    }


    # boot if missing either mcc or mnc.
    next if ($mcc eq "" or $mnc eq "");

    # try to covert diacritic chars to ascii in operator_name.
    my $operator_name = NFD($operator_name);
    $operator_name =~ s/\pM+//g;
    $operator_name = unidecode($operator_name);

    if ($operator_name =~ /^\s*$/) {
	$operator_name = "unknown";
    }
    $operator_name =~ s/A3/o/g;
    $operator_name =~ s/A\+\-/n/g;
    $operator_name =~ s/A\!/s/g;
    $operator_name =~ s/A1\/2/Z/g;
    $operator_name =~ s/AmA\(c\)/A/g;
    $operator_name =~ s/Ao/u/g;
    $operator_name =~ s/\[|\]//g;


    unless ($mcc eq "" and $mnc eq "" and $operator_name eq "") {
	unless (defined $country and $country ne "") {
	    $country = 'ZZ';
	}
	print OFD "$mcc\t$mnc\t$operator_name\t$country\n";
    }
}

close OFD;
